package net.oov.identify.support;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;

import net.oov.identify.Dictionary;
import net.oov.identify.OI;
import net.oov.identify.SequenceFreqContainerFactory;


/**
 * 未登录词数量统计
 * 词典匹配的串数记为登录词数
 * 词典未匹配且汉字长大于1的串数记为未登录词数量(分词碎片连续个数大于2（包含2）的频次)
 * @author huangsui.cn
 *
 */
public class OOVAmountStatistics {
	private String rawTextPath = "txt/in_raw_50026637.txt";
	private Integer OOVAmount = 0;
	private Integer wordsAmount = 0;
	
	/**
	 * 
	 * 未登录词数量统计
	 * @param line
	 * @param dic
	 * @return
	 */
	private void statistics(String line, Dictionary dic){
		
		SequenceFreqContainerFactory sfcf = new SequenceFreqContainerFactory();
		line = sfcf.specialCharFilter(line);
		String[] str = line.split(" ");
		String str_ = null;
		for(int i=0;i<str.length;i++){
			str_ = str[i];
			if(str_.trim().length()<1)continue;
			int[] indexs =  dic.revMaxFirstMatch(str_, 6);//[1,0,0,1,1,1,1,0]
			int k = 0;//标记单字的起始,标记indexs上次值为1的下标
			int num = 0;//记录连续单字的个数
			for(int j=1;j<indexs.length;j++){
				if(indexs[j]==0){
					if(num>=2){
						OOVAmount++;
						num = 0;
					}
					if(j==k+1){
						wordsAmount++;
					}
					continue;//下面indexs[j]==1
				}
				if(j==k+1){//j比k大1，说明是单字词
					num++;
					if(j==indexs.length-1){
						num++;
						break;
					}
					
				}
				k = j;
			}
			if(num>=2)OOVAmount++;
		}
	
	}
	
	/**
	 * 
	 * @param sfc
	 * @param reader
	 * @return
	 * 
	 */
	public void test(){
		BufferedReader br;
		try {
			br = new BufferedReader(new InputStreamReader(new FileInputStream(rawTextPath),"UTF-8"));
		} catch (Exception e) {
			throw new RuntimeException(e);
		}
		Dictionary dic = null;//TODO:singleton
		String line = null;
		try {
			while((line = br.readLine())!=null){
				statistics(line, dic);
			}
		} catch (IOException e) {
			throw new RuntimeException(e);
		}
		
		OI.wl("wordsAmount:"+wordsAmount+",OOVAmount:"+OOVAmount);
		
	}
	
	
	public static void main(String[] args){
		OOVAmountStatistics as = new OOVAmountStatistics();
		as.test();
	}

}
